
from google.colab import drive
drive.mount('/content/drive')
%cd '/content/drive/My Drive/Colab Notebooks/imageCaptions'

!nvidia-smi

Mounted at /content/drive
/content/drive/My Drive/Colab Notebooks/imageCaptions
Thu Nov 26 15:27:03 2020       
+-----------------------------------------------------------------------------+
| NVIDIA-SMI 455.38       Driver Version: 418.67       CUDA Version: 10.1     |
|-------------------------------+----------------------+----------------------+
| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |
| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |
|                               |                      |               MIG M. |
|===============================+======================+======================|
|   0  Tesla K80           Off  | 00000000:00:04.0 Off |                    0 |
| N/A   71C    P8    32W / 149W |      0MiB / 11441MiB |      0%      Default |
|                               |                      |                 ERR! |
+-------------------------------+----------------------+----------------------+
                                                                               
+-----------------------------------------------------------------------------+
| Processes:                                                                  |
|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |
|        ID   ID                                                   Usage      |
|=============================================================================|
|  No running processes found                                                 |
+-----------------------------------------------------------------------------+


import numpy as np
from matplotlib import pyplot as plt
import os
import cv2
from pprint import pprint

import torch
from tqdm import tqdm_notebook as tqdm
import torch.nn as nn
import torch.optim as optim
import torchvision.transforms as transforms
from torch.utils.tensorboard import SummaryWriter
from utils import save_checkpoint, load_checkpoint, print_examples
from MyDataloader import get_loader as my_get_loader
from model import EncoderDecoder
from model import *
from nltk.translate.bleu_score import sentence_bleu, SmoothingFunction

device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
ROOT_FOLDER = "/content/drive/My Drive/Colab Notebooks/imageCaptions/flickr8k/images"


def disp_msg(split_type, length):
    print(f'{split_type} dataset: Total number of images {length} \nwith a total number of recordes {length*5}', end='\n')


from collections import defaultdict
import json
import pandas as pd
from glob import glob

#Read the images 
actuall_images = glob('images/*')
actuall_images = [img.split('/')[-1] for img in actuall_images]

# for getting the split to train, test and validation Let's read the JSON file, and extract the data from it.
dataset_path = r'/content/drive/MyDrive/Colab Notebooks/imageCaptions/flickr8k/dataset.json'
dataset = json.load(open(dataset_path, 'r', encoding="utf8"))
split = defaultdict(list)
for img in dataset['images']:
    split[img['split']].append(img)


names = ['train', 'val', 'test']
total_images_from_dataset = 0
for name in names:
    split_values = split[f'{name}']
    df = pd.DataFrame.from_dict(split_values) 
    images_list = df['filename'].tolist()
    print("\n")
    display(df.head(10))
    print("\n")
    disp_msg(name, len(split_values))
    print("\n")


train dataset: Total number of images 6000 
with a total number of recordes 30000


val dataset: Total number of images 1000 
with a total number of recordes 5000


test dataset: Total number of images 1000 
with a total number of recordes 5000


df = pd.DataFrame.from_dict(split['val']) 
for sentence in df.iloc[0]['sentences']:
    pprint(sentence)
    print('\n')

{'imgid': 6000,
 'raw': 'the boy laying face down on a skateboard is being pushed along the '
        'ground by another boy .',
 'sentid': 30000,
 'tokens': ['the',
            'boy',
            'laying',
            'face',
            'down',
            'on',
            'a',
            'skateboard',
            'is',
            'being',
            'pushed',
            'along',
            'the',
            'ground',
            'by',
            'another',
            'boy']}


{'imgid': 6000,
 'raw': 'Two girls play on a skateboard in a courtyard .',
 'sentid': 30001,
 'tokens': ['two',
            'girls',
            'play',
            'on',
            'a',
            'skateboard',
            'in',
            'a',
            'courtyard']}


{'imgid': 6000,
 'raw': 'Two people play on a long skateboard .',
 'sentid': 30002,
 'tokens': ['two', 'people', 'play', 'on', 'a', 'long', 'skateboard']}


{'imgid': 6000,
 'raw': 'Two small children in red shirts playing on a skateboard .',
 'sentid': 30003,
 'tokens': ['two',
            'small',
            'children',
            'in',
            'red',
            'shirts',
            'playing',
            'on',
            'a',
            'skateboard']}


{'imgid': 6000,
 'raw': 'two young children on a skateboard going across a sidewalk',
 'sentid': 30004,
 'tokens': ['two',
            'young',
            'children',
            'on',
            'a',
            'skateboard',
            'going',
            'across',
            'a',
            'sidewalk']}


def show_img(image_id):
    plt.figure(figsize=[15, 9])
    img = plt.imread(os.path.join(ROOT_FOLDER, image_id))
    img = cv2.resize(img, (600, 500))
    eps = 20 
    df = pd.DataFrame.from_dict(split['val']) 
    eps = 0
    for sentence in df.iloc[0]['sentences']:
        plt.text(2, 10 + eps, sentence['raw'], fontweight='bold', fontsize=12, bbox=dict(facecolor='green', alpha=0.5))
        eps += 20
    plt.imshow(img)
    plt.axis('off')
    plt.show()

    
show_img('2090545563_a4e66ec76b.jpg')


def clean_sentence(sentence):
    for i, item in enumerate(sentence):
        if item == '<SOS>':
            sentence.remove(item)

        if item == '<EOS>':
            del sentence[i: len(sentence)] # also remove post [<PAD>] tokens
    return sentence

def blue_score(batch, dataset, show_output=False):
    n_grams_result = {
            '1-gram':[],
            '2-gram':[],
            '3-gram':[],
            '4-gram':[]
        }
    imgs, captions, imgs_id= batch[:3]
    smoothie = SmoothingFunction().method1
    results = {
            'image id':[],
            'kpi': [],
            'captions list': [],
            'output': []
    }
    with torch.no_grad():                 
        for i, img in enumerate(imgs):
            
            img = img.unsqueeze(0)
            captions_list = [clean_sentence([dataset.vocab.itos[idx.detach().cpu().item()] for idx in captions[:, i, j]]) for j in range(5)]
            output = clean_sentence(model.caption(img.to(device), train_dataset.vocab))

            gram1 = sentence_bleu(captions_list, output, weights=(1, 0, 0, 0), smoothing_function=smoothie)
            gram2 = sentence_bleu(captions_list, output, weights=(0.5, 0.5, 0, 0), smoothing_function=smoothie)
            gram3 = sentence_bleu(captions_list, output, weights=(0.33, 0.33, 0.33, 0), smoothing_function=smoothie)
            gram4 = sentence_bleu(captions_list, output, weights=(0.25, 0.25, 0.25, 0.25), smoothing_function=smoothie)

            n_grams_result['1-gram'].append(gram1)
            n_grams_result['2-gram'].append(gram2)
            n_grams_result['3-gram'].append(gram3)
            n_grams_result['4-gram'].append(gram4)
            
            results['image id'].append(imgs_id[i])
            results['kpi'].append(
                {
                    '1-gram': gram1,
                    '2-gram': gram2,
                    '3-gram': gram3,
                    '4-gram': gram4
                }
            )
            results['captions list'].append(captions_list)
            results['output'].append(output)

            if show_output:
                print(f'Output: {" ".join(output)}')
                for i, cap in enumerate(captions_list):
                    print(f'option {i+1}: {" ".join(cap)}')
                print('\n')


        n_grams_result['1-gram'] = np.mean(np.array(n_grams_result['1-gram']))
        n_grams_result['2-gram'] = np.mean(np.array(n_grams_result['2-gram']))
        n_grams_result['3-gram'] = np.mean(np.array(n_grams_result['3-gram']))
        n_grams_result['4-gram'] = np.mean(np.array(n_grams_result['4-gram']))
        if show_output:
            print(n_grams_result)
            print('\n')
    return n_grams_result, results


transform = transforms.Compose(
    [
        transforms.Resize(size=(224, 224)),
        transforms.ToTensor(),
        transforms.Normalize((0.485, 0.456, 0.406),
                                (0.229, 0.224, 0.225))
    ])


train_loader, train_dataset = my_get_loader(
                                    root_folder=ROOT_FOLDER,
                                    df= pd.DataFrame.from_dict(split['train']),
                                    transform=transform,
                                    batch_size=32,
                                    num_workers=4,
                                    shuffle=True)

val_loader, val_dataset = my_get_loader(
                                    root_folder=ROOT_FOLDER,
                                    df= pd.DataFrame.from_dict(split['val']),
                                    transform=transform,
                                    batch_size=32,
                                    num_workers=4,
                                    shuffle=False)


torch.backends.cudnn.benchmark = True

 
load_model, train_CNN, save_model = False, False, True

# architecture hyperparameters set ups
embed_size, hidden_size, num_layers = 512, 512, 2
vocab_size = len(train_dataset.vocab)

# train hyperparameters
learning_rate = 1e-3
num_epochs = 150

# set tensorboard writer
writer = SummaryWriter(f'runs/run_for_graphs1')

# architecture
model =  EncoderDecoder(embed_size, hidden_size, vocab_size, num_layers).to(device)

# loss function
criterion = nn.CrossEntropyLoss(ignore_index=train_dataset.vocab.stoi["<PAD>"]) 

# optimizer
optimizer =  optim.Adam(model.parameters(), lr=learning_rate)
scheduler = None


def train_one_epoch(model, optimizer, dataloader, dataset, step, writer):
    kpi={
       'loss': [],
       '1-gram':[],
       '2-gram':[],
       '3-gram':[],
       '4-gram':[]
    }
    model.train()
    for idx, data in tqdm(
        enumerate(dataloader), total=len(dataloader)):
        ''' imgs shape [32, 3, 224, 224]
            captions shape [max sentence length, 32, 5] '''
        imgs, captions, _ = data[:3]
        max_length, batch_size, _ = captions.shape
        captions = captions.reshape(max_length, batch_size*5)

        imgs = imgs.to(device)
        captions = captions.to(device)
        imgs = torch.repeat_interleave(imgs, repeats=5, dim=0) #[a, b, c] -> [aaaaa, bbbbb, ccccc].T

        ''' imgs shape [160, 3, 224, 244]
            captions shape [max sentence length, 160] '''
        outputs = model(imgs, captions[:-1])

        loss = criterion(
            outputs.reshape(-1, outputs.shape[2]), captions.reshape(-1)
        )

        writer.add_scalar("Training loss", loss.item(), global_step=step)
        if idx == (len(dataloader) -1):
            print('\nTrain result\n')
            n_grams_result, _ = blue_score(data, dataset, show_output=True) #calculate blue score of the batch
        else:
            n_grams_result, _ = blue_score(data, dataset) #calculate blue score of the batch
        writer.add_scalar("1-gram bleu_train", n_grams_result['1-gram'] , global_step=step)
        writer.add_scalar("2-gram bleu_train", n_grams_result['2-gram'] , global_step=step)
        writer.add_scalar("3-gram bleu_train", n_grams_result['3-gram'] , global_step=step)
        writer.add_scalar("4-gram bleu_train", n_grams_result['4-gram'] , global_step=step)
        step += 1

        optimizer.zero_grad()
        loss.backward(loss)
        optimizer.step()

        kpi['loss'].append(loss.item())
        kpi['1-gram'].append(n_grams_result['1-gram'])
        kpi['2-gram'].append(n_grams_result['2-gram'])
        kpi['3-gram'].append(n_grams_result['3-gram'])
        kpi['4-gram'].append(n_grams_result['4-gram'])

    return model, optimizer, kpi, step, writer
for data in train_loader:
    imgs, captions, _ = data[:3]
    print(f'images shape: {imgs.shape}')
    print(f'captions shape: {captions.shape}')
    break

images shape: torch.Size([32, 3, 224, 224])
captions shape: torch.Size([27, 32, 5])


def val_one_epoch(model, dataloader, dataset, step, writer):
    kpi={
       'loss': [],
       '1-gram':[],
       '2-gram':[],
       '3-gram':[],
       '4-gram':[]
    }
    model.eval()
    with torch.no_grad():
        for idx, data in tqdm(
        enumerate(dataloader), total=len(dataloader)):

            ''' imgs shape [32, 3, 224, 244]
            captions shape [max sentence length, 32, 5] '''

            imgs, captions, imgs_id = data[:3]
            max_length, batch_size, _ = captions.shape
            captions = captions.reshape(max_length, batch_size*5)

            imgs = imgs.to(device)
            captions = captions.to(device)
            imgs = torch.repeat_interleave(imgs, repeats=5, dim=0)

            ''' imgs shape [160, 3, 224, 244]
            captions shape [max sentence length, 160] '''

            outputs = model(imgs, captions[:-1])

            loss = criterion(
                outputs.reshape(-1, outputs.shape[2]), captions.reshape(-1)
            )
            if idx == (len(dataloader) -1):
                print('\nValidition result\n')
                n_grams_result, _ = blue_score(data, dataset, show_output=True) #calculate blue score of the batch
            else:
                n_grams_result, _ = blue_score(data, dataset) #calculate blue score of the batch

            writer.add_scalar("Validition loss", loss.item(), global_step=step)
            writer.add_scalar("1-gram bleu_val", n_grams_result['1-gram'] , global_step=step)
            writer.add_scalar("2-gram bleu_val", n_grams_result['2-gram'] , global_step=step)
            writer.add_scalar("3-gram bleu_val", n_grams_result['3-gram'] , global_step=step)
            writer.add_scalar("4-gram bleu_val", n_grams_result['4-gram'] , global_step=step)
            step += 1

            kpi['loss'].append(loss.item())
            kpi['1-gram'].append(n_grams_result['1-gram'])
            kpi['2-gram'].append(n_grams_result['2-gram'])
            kpi['3-gram'].append(n_grams_result['3-gram'])
            kpi['4-gram'].append(n_grams_result['4-gram'])

        return model, kpi, step, writer
for data in val_loader:
    imgs, captions, _ = data[:3]
    print(f'images shape: {imgs.shape}')
    print(f'captions shape: {captions.shape}')
    break

images shape: torch.Size([32, 3, 224, 224])
captions shape: torch.Size([34, 32, 5])


# save_point = torch.load("/content/drive/MyDrive/Colab Notebooks/imageCaptions/run_for_graphs.pth.tar")


# # def load_checkpoint(checkpoint, model, optimizer):
#     # print("=> Loading checkpoint")
# train_step = 0
# val_step = 0

# model.load_state_dict(save_point["state_dict"])
# optimizer.load_state_dict(save_point["optimizer"])
# train_step = save_point["train_step"]
# val_step = save_point["val_step"]
#     # return train_step, val_step
# num_epochs = num_epochs - int(train_step/len(train_loader))


train_step = 0
val_step = 0

for epoch in tqdm(range(num_epochs), total=num_epochs):
    print(f'epoch: [{epoch+1}/{num_epochs}]', end='\n')
    print_examples(model, device, train_dataset, 224)

    model.train()
    model, optimizer, train_kpi, train_step, writer = train_one_epoch(model, optimizer, train_loader, train_dataset, train_step, writer)
    model, val_kpi, val_step, writer = val_one_epoch(model, val_loader, val_dataset, val_step, writer)
    
    if save_model:
        checkpoint = {
            "state_dict": model.state_dict(),
            "optimizer": optimizer.state_dict(),
            "train_step": train_step,
            "val_step": val_step,
            'train_kpi': train_kpi,
            'val_kpi': val_kpi
        }
        save_checkpoint(checkpoint, filename="run_for_graphs.pth.tar")

    # %reload_ext tensorboard
    # %tensorboard --logdir {LOG_DIR}
    # if scheduler:
    #     scheduler.step()
    
writer.close()

/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:5: TqdmDeprecationWarning: This function will be removed in tqdm==5.0.0
Please use `tqdm.notebook.tqdm` instead of `tqdm.tqdm_notebook`
  """

epoch: [1/99]
Example 1 CORRECT: Dog on a beach by the ocean
Example 1 OUTPUT: <SOS> a man and a dog on a dock are standing in the ocean . <EOS>
Example 2 CORRECT: Child holding red frisbee outdoors
Example 2 OUTPUT: <SOS> a young boy wearing a red shirt and blue jeans is running through a field of flowers . <EOS>
Example 3 CORRECT: Bus driving by parked cars
Example 3 OUTPUT: <SOS> a man and a woman are walking past a record of a car . <EOS>
Example 4 CORRECT: A small boat in the ocean
Example 4 OUTPUT: <SOS> a man is fishing in a lake . <EOS>
Example 5 CORRECT: A cowboy riding a horse in the desert
Example 5 OUTPUT: <SOS> a man and a dog on a dock are standing in the ocean . <EOS>

/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:12: TqdmDeprecationWarning: This function will be removed in tqdm==5.0.0
Please use `tqdm.notebook.tqdm` instead of `tqdm.tqdm_notebook`
  if sys.path[0] == '':

Train result

Output: a man in a white helmet and sunglasses is sitting on a bench .
option 1: a dog jumps into the pond .
option 2: a dog leaping into the water near a <UNK> .
option 3: a light colored dog jumping off the back of a boat into the water .
option 4: a tan dog jumping off of a boat into the water .
option 5: the yellow dog is jumping off of a ship into the water .


Output: a man in a white helmet holds a <UNK> .
option 1: a brown and black dog by a tree
option 2: a brown , black , and white dog barking up a tree .
option 3: a dog is perched up against a tree looking at the top .
option 4: a dog looking up a tree with his tongue hanging out .
option 5: a dog with its tongue out is trying to look up the tree .


Output: a man in a white helmet holds a <UNK> .
option 1: a black dog is walking along the side of a pool .
option 2: a black dog runs around an outdoor swimming pool .
option 3: a black dog walking beside a pool .
option 4: a black poodle walks on the edge of a pool .
option 5: a fluffy <UNK> poodle is walking on the edge of a pool .


Output: a man in a white helmet holds a <UNK> .
option 1: a woman is looking at dressed , <UNK> <UNK> in a store display .
option 2: a woman looking at clothing on a <UNK> .
option 3: a woman looks at some <UNK> in a window display .
option 4: a woman is looking at a display case with three outfits on a <UNK> of bright yellow .
option 5: woman looking at dresses in window of a clothing store .


Output: a man in a white helmet holds a <UNK> .
option 1: a bicycler does a trick in the air and his shadow is <UNK> on the ceiling .
option 2: a boy doing a trick in the air on his bicycle
option 3: a man in a brown shirt and jeans is doing a trick on his bicycle .
option 4: a man performing a bicycle trick inside in midair .
option 5: the man is doing a trick on a bike in midair .


Output: a man in a white helmet holds a <UNK> .
option 1: two males are playing field hockey , one with a blue stick , one with a gold stick ,
option 2: two men are playing in a sports game together .
option 3: two men playing a game with some sort of sticks .
option 4: two men playing a sport are fighting for a ball .
option 5: two players <UNK> for the ball in field hockey .


Output: a man in a white helmet holds a <UNK> .
option 1: four dogs are playing and jumping in the air outside .
option 2: four dogs jumping in the air in a field .
option 3: four dogs look at an object in the sky , one jumping to reach it .
option 4: four dogs watch an unseen object , two of them leaping in the air .
option 5: two dogs on hind legs and two seated dogs are looking at something in the air .


Output: a man in a white helmet holds a <UNK> .
option 1: a guy in a costume , a crowd watching him .
option 2: a man standing in front of a crowd .
option 3: a performer putting on an act in front of a large crowd of people .
option 4: a person wearing a black coat and a helmet with <UNK> things is posing on the lawn in front of a crowd of people .
option 5: one person performing a show in front of a large audience sitting on the lawn


Output: a man in a white helmet holds a <UNK> .
option 1: people with equipment climbing up a snowy mountain
option 2: people are hiking up a mountain in the snow .
option 3: two hikers are backpacking up a snow slope .
option 4: two hikers with backpacks crossing a snow field .
option 5: two <UNK> are walking across a snowcapped mountain .


Output: a man in a white helmet and sunglasses is sitting on a bench .
option 1: a fisherman stands on the beach on a gray day .
option 2: a man holding a fishing pole and tackle box walking in from the ocean .
option 3: a man in a yellow cap is on the beach carrying a fishing pole .
option 4: a man with a fishing pole standing on a beach .
option 5: a person is standing in the ocean <UNK> <UNK> holding a fishing pole in one hand and a tackle box in the other .


Output: a man in a white helmet holds a <UNK> .
option 1: children playing around a <UNK>
option 2: four children are dancing around a pole in a city street .
option 3: four children playing around a pole .
option 4: four children running around a pole with red and white <UNK> on top .
option 5: little kids play a game of running around a pole .


Output: a man in a white helmet holds a <UNK> .
option 1: a red airplane is leaving white smoke behind it .
option 2: a red airplane that has left behind a trail .
option 3: a red <UNK> in the sky , with a trail of smoke .
option 4: a red <UNK> - plane leaves a white trail in a blue sky .
option 5: a red <UNK> <UNK> across the sky leaving a white trail behind it .


Output: a man in a white helmet holds a <UNK> .
option 1: a black haired girl is eating corn on the <UNK> .
option 2: a man is eating a piece of corn on the <UNK> .
option 3: a man is <UNK> on an ear of corn
option 4: a person with dark hair is eating a very large ear of corn .
option 5: the child with black hair is eating corn on the <UNK> .


Output: a man in a white helmet holds a <UNK> .
option 1: two men are wrestling in a competition .
option 2: two men are wrestling while other people are watching
option 3: two men wrestle on the floor in front of an group of young men .
option 4: two men wrestling at a match .
option 5: two young men competing in a wrestling tournament .


Output: a man in a white helmet holds a <UNK> .
option 1: a bicycle goes downhill .
option 2: a bicyclist wearing a dark helmet and a blue jacket jumps over a mound in a field .
option 3: a man in a blue jacket riding a bike
option 4: a man is biking on a hill .
option 5: a person rides their bike down a hill .


Output: a man in a white helmet holds a <UNK> .
option 1: an elderly woman is standing by the window of a grey stone building .
option 2: a person standing on the sidewalk , next to a window , wearing warm clothes
option 3: a woman all dressed in blue with blue bags stands outside a window .
option 4: a woman in a fur hat and fur coat stands outside a building .
option 5: the elderly lady waits on the sidewalk .


{'1-gram': 0.5141973184323136, '2-gram': 0.2744127729408252, '3-gram': 0.14660257166557406, '4-gram': 0.09737339597975145}

Validition result

Output: a man on a motorcycle is racing on a track .
option 1: a man in a <UNK> <UNK> outfit is sitting on a <UNK> motorcycle .
option 2: a man in wearing a mask and black leather is sitting on a motorcycle on the street .
option 3: a person on a motorcycle wearing full leather with a <UNK> on it and <UNK> mask .
option 4: man in black leather on a motorcycle .
option 5: man in <UNK> - <UNK> leather outfit sitting <UNK> a red motorcycle .


Output: a man is fishing in the ocean with a sunset in the background .
option 1: a brown dog <UNK> up a <UNK> from a stone surface .
option 2: a brown puppy chewing on a stick .
option 3: a dog with big ears is holding a small stick in its mouth .
option 4: a puppy has a stick in its mouth .
option 5: a puppy with big ears chewing on a stick .


Output: a man in a yellow shirt is surfing on a white surfboard .
option 1: a man <UNK> , <UNK> his off hand into the water .
option 2: a person leaning over while <UNK> .
option 3: a shirtless man is leaning back while <UNK> .
option 4: a <UNK> <UNK> the water with his <UNK> hand .
option 5: a young man <UNK> in the ocean .


Output: a man is fishing in the ocean with a boat on the river
option 1: a lone bicyclist jumping on his bike in front of a <UNK> wall .
option 2: a man doing tricks on a bike
option 3: a man <UNK> the <UNK> of his bicycle while he jumps .
option 4: a man on a bicycle in jeans is doing an air track along a dirt track with graffiti nearby .
option 5: a man performs bike tricks by a <UNK> .


Output: a man is fishing in the ocean with a boat on the river
option 1: a black dog running along the sandy <UNK> of a body of water while two ducks swim nearby .
option 2: a black dog running along the water
option 3: a dog and two birds .
option 4: a dog runs along the shore of a pond with two <UNK> <UNK> swimming .
option 5: dog running on shore beside two <UNK> in water


Output: a man is standing on a shore looking out at the ocean .
option 1: people kayaking in snowy and <UNK> <UNK> .
option 2: snowy mountains are in the background of three ocean kayakers .
option 3: three kayakers are traveling in the water alongside <UNK> mountains .
option 4: three men kayaking with mountains in the background
option 5: three people on red boats paddle through water with mountains in the background .


Output: a man is sitting on a bench with his bike and looking at the view .
option 1: a girl in pink pants running , an <UNK> <UNK> <UNK> in the background .
option 2: the girl dressed in purple walks through the <UNK> , <UNK> by a girl in pink .
option 3: two children <UNK> the <UNK> of a building .
option 4: two girls , one running away from the wall .
option 5: two little girls play around an old <UNK> building .


Output: a baby is peeking over the top of a melting playpen onto a human bed .
option 1: a girl playing is a pile of colorful <UNK> .
option 2: a little girl plays in a ball <UNK> .
option 3: a little girl plays in a <UNK> of colorful <UNK> .
option 4: a small girl is playing in a ball <UNK>
option 5: a young girl with a white shirt and pink shorts rolling around in some plastic <UNK> <UNK>


{'1-gram': 0.40905443450160717, '2-gram': 0.13178320216285735, '3-gram': 0.0668596902837546, '4-gram': 0.037261533141678155}


=> Saving checkpoint
epoch: [2/99]
Example 1 CORRECT: Dog on a beach by the ocean
Example 1 OUTPUT: <SOS> a dog runs through the water . <EOS>
Example 2 CORRECT: Child holding red frisbee outdoors
Example 2 OUTPUT: <SOS> a young boy in a striped shirt and jeans runs in the grass . <EOS>
Example 3 CORRECT: Bus driving by parked cars
Example 3 OUTPUT: <SOS> a man in a white coat and hat stands next to a column . <EOS>
Example 4 CORRECT: A small boat in the ocean
Example 4 OUTPUT: <SOS> a man is fishing in the ocean with a sunset in the background . <EOS>
Example 5 CORRECT: A cowboy riding a horse in the desert
Example 5 OUTPUT: <SOS> a dog runs through the woods . <EOS>

Train result

Output: a man in a white shirt and jeans is <UNK> a white dog to jump over a pole .
option 1: a couple of people <UNK> through a muddy pit
option 2: a person wearing orange pants in playing in the mud .
option 3: two girls climbing out of a mud pit by a beach .
option 4: two girls wading through <UNK> deep mud .
option 5: two girls <UNK> through some mud at a park .


Output: a man in a white shirt and cap is <UNK> a white dog 's face .
option 1: a bunch of football players , and a crowd .
option 2: a football game .
option 3: a large group of football players about to start a game with a large group watching
option 4: two football teams at the line of <UNK>
option 5: two football teams lined up before the hike .


Output: a man in a white shirt and jeans is <UNK> a white dog to jump over a pole .
option 1: a dog chases a smaller dog .
option 2: a large dog is running on the grass .
option 3: a larger brown dog is chasing a smaller grey dog on the grass .
option 4: a large tan dog runs after a small tan dog on a grassy yard .
option 5: a small dog stops in its tracks to look at the big dog chasing it .


Output: a man in a white shirt and jeans is <UNK> a white dog to jump over a pole .
option 1: two ladies posing for a picture
option 2: two middle aged <UNK> woman standing next to each other and smiling at the camera .
option 3: two middle aged women stand for a picture near a forest .
option 4: two older women smiling and standing in front of large trees , one woman is <UNK> her hair .
option 5: two women with long hair and large - <UNK> <UNK> are posing for a camera in front of trees .


Output: a man in a white shirt and cap is <UNK> a white dog 's face .
option 1: a boy gets <UNK> into the river by another boy .
option 2: a group of children play in the water under a bridge .
option 3: an airborne kid is watched by his <UNK> and others .
option 4: boys jump off a bridge into the water .
option 5: man in white shirt flipping young boy in the water with four other boys surrounding them .


Output: a man in a white shirt and cap is <UNK> a white dog 's face .
option 1: children playing outside in a fountain .
option 2: two boys in underwear play in the water in the city street .
option 3: two little boys play in the water left behind by the sprinklers .
option 4: two young boys are playing in a fountain in a city plaza .
option 5: two young boys play in the water that has <UNK> on the street .


Output: a man in a white shirt and cap is <UNK> a white dog 's face .
option 1: a dirt biker participates in a race .
option 2: a dirt bike <UNK> up the terrain .
option 3: a man on a bike throws up dirt during a motor cross event .
option 4: a motocross bike kicks up dirt as it is being ridden around a bend in the circuit .
option 5: a person in a blue outfit rides a motorcycle through a hill of dirt .


Output: a man in a white shirt and jeans is <UNK> a white dog to jump over a pole .
option 1: a group of people are riding in the back of a truck .
option 2: a group of warmly dressed people in the back of a truck .
option 3: a <UNK> of <UNK> in the back of a large wagon .
option 4: many middle eastern people are sitting in the back of a blue truck .
option 5: people are riding in a crowded truck bed .


Output: a man in a white shirt and jeans is <UNK> a white dog to jump over a pole .
option 1: four shirtless men are hiking up a canyon .
option 2: four shirtless men hike up a canyon .
option 3: shirtless men hike through a canyon .
option 4: the hikers are climbing up the mountains .
option 5: topless men walking up a rocky path .


Output: a man in a white shirt and cap is <UNK> a white dog 's face .
option 1: a group of children wearing blue and yellow uniforms put their hangs together .
option 2: a soccer team with blue jerseys is huddled together with their hands touching above their heads .
option 3: a young flag football team are huddled together .
option 4: boys huddle during a soccer game .
option 5: soccer players in blue before a game .


Output: a man in a white shirt and jeans is <UNK> a white dog to jump over a pole .
option 1: a dog in water holds a stick in its mouth .
option 2: a german shepherd is running with a stick in its mouth .
option 3: a large yellow dog is retrieving a stick .
option 4: a tan dog holding a stick in his mouth .
option 5: the yellow dog is carrying a stick by water .


Output: a man in a white shirt and jeans is <UNK> a white dog to jump over a pole .
option 1: a dark tent with <UNK> lighting and people dancing with a man in a suit in the foreground .
option 2: a group of people in a dimly lit night club .
option 3: a man looks on in a club .
option 4: a man stares out a window .
option 5: people are gathering under colored lights in a large tent .


Output: a man in a white shirt and cap is <UNK> a white dog 's face .
option 1: a fisherman stands on the beach on a gray day .
option 2: a man holding a fishing pole and tackle box walking in from the ocean .
option 3: a man in a yellow cap is on the beach carrying a fishing pole .
option 4: a man with a fishing pole standing on a beach .
option 5: a person is standing in the ocean <UNK> <UNK> holding a fishing pole in one hand and a tackle box in the other .


Output: a man in a white shirt and jeans is <UNK> a white dog to jump over a pole .
option 1: a brown dog playing with a tennis ball
option 2: a brown dog reaches for a tennis ball .
option 3: a gray dog is looking at a ball in the air .
option 4: a large skinny dog is trying to catch a <UNK> green tennis ball .
option 5: brown dog catches a ball in a sunny yard


Output: a man in a white shirt and cap is <UNK> a white dog 's face .
option 1: a woman dressed in black is <UNK> outside on a grassy lawn with a leashed black and white dog while people in the background watch .
option 2: a woman in gray holding onto the black dog 's leash .
option 3: a woman <UNK> with a pair of dogs .
option 4: a woman showing a long haired , large black <UNK> dog at a dog show .
option 5: a woman shows a dog with very long hair at a dog show .


Output: a man in a white shirt and cap is <UNK> a white dog 's face .
option 1: the children play in the pool .
option 2: two children are playing in an outdoor swimming pool .
option 3: two children playing in a swimming pool .
option 4: two children play in the water of an above - ground pool .
option 5: two little girls playing in the pool .


{'1-gram': 0.3874861683597534, '2-gram': 0.15636482520457476, '3-gram': 0.07173072815140233, '4-gram': 0.046476621281160854}

Validition result

Output: a man in a red shirt is riding a bicycle on a track .
option 1: a man in a <UNK> <UNK> outfit is sitting on a <UNK> motorcycle .
option 2: a man in wearing a mask and black leather is sitting on a motorcycle on the street .
option 3: a person on a motorcycle wearing full leather with a <UNK> on it and <UNK> mask .
option 4: man in black leather on a motorcycle .
option 5: man in <UNK> - <UNK> leather outfit sitting <UNK> a red motorcycle .


Output: a man is standing on a shore overlooking the ocean .
option 1: a brown dog <UNK> up a <UNK> from a stone surface .
option 2: a brown puppy chewing on a stick .
option 3: a dog with big ears is holding a small stick in its mouth .
option 4: a puppy has a stick in its mouth .
option 5: a puppy with big ears chewing on a stick .


Output: a man is standing in front of a mountain surrounded by green hills .
option 1: a man <UNK> , <UNK> his off hand into the water .
option 2: a person leaning over while <UNK> .
option 3: a shirtless man is leaning back while <UNK> .
option 4: a <UNK> <UNK> the water with his <UNK> hand .
option 5: a young man <UNK> in the ocean .


Output: a man is standing on a shore overlooking the ocean .
option 1: a lone bicyclist jumping on his bike in front of a <UNK> wall .
option 2: a man doing tricks on a bike
option 3: a man <UNK> the <UNK> of his bicycle while he jumps .
option 4: a man on a bicycle in jeans is doing an air track along a dirt track with graffiti nearby .
option 5: a man performs bike tricks by a <UNK> .


Output: a man is standing on a shore overlooking the ocean .
option 1: a black dog running along the sandy <UNK> of a body of water while two ducks swim nearby .
option 2: a black dog running along the water
option 3: a dog and two birds .
option 4: a dog runs along the shore of a pond with two <UNK> <UNK> swimming .
option 5: dog running on shore beside two <UNK> in water


Output: a man in a white shirt is standing in front of a mountain surrounded by green hills .
option 1: people kayaking in snowy and <UNK> <UNK> .
option 2: snowy mountains are in the background of three ocean kayakers .
option 3: three kayakers are traveling in the water alongside <UNK> mountains .
option 4: three men kayaking with mountains in the background
option 5: three people on red boats paddle through water with mountains in the background .


Output: a man in a white shirt and jeans is walking down a sidewalk .
option 1: a girl in pink pants running , an <UNK> <UNK> <UNK> in the background .
option 2: the girl dressed in purple walks through the <UNK> , <UNK> by a girl in pink .
option 3: two children <UNK> the <UNK> of a building .
option 4: two girls , one running away from the wall .
option 5: two little girls play around an old <UNK> building .


Output: a small dog is playing with a toy .
option 1: a girl playing is a pile of colorful <UNK> .
option 2: a little girl plays in a ball <UNK> .
option 3: a little girl plays in a <UNK> of colorful <UNK> .
option 4: a small girl is playing in a ball <UNK>
option 5: a young girl with a white shirt and pink shorts rolling around in some plastic <UNK> <UNK>


{'1-gram': 0.4656277486772231, '2-gram': 0.2442037284891723, '3-gram': 0.10765667775845487, '4-gram': 0.06930117128711354}


=> Saving checkpoint
epoch: [3/99]
Example 1 CORRECT: Dog on a beach by the ocean
Example 1 OUTPUT: <SOS> a dog lies on its back in the sand . <EOS>
Example 2 CORRECT: Child holding red frisbee outdoors
Example 2 OUTPUT: <SOS> a little girl in a pink shirt and jeans is running through a flowered field . <EOS>
Example 3 CORRECT: Bus driving by parked cars
Example 3 OUTPUT: <SOS> a man in a white shirt and jeans stands next to a column in front of a store . <EOS>
Example 4 CORRECT: A small boat in the ocean
Example 4 OUTPUT: <SOS> a man is standing on rocks by the water and away from beautiful mountains . <EOS>
Example 5 CORRECT: A cowboy riding a horse in the desert
Example 5 OUTPUT: <SOS> a man is standing in front of a waterfall . <EOS>

Train result

Output: a man in a white helmet is walking with a dog .
option 1: a boy on a snowboard leaps through the air .
option 2: a man in a yellow hat is crouching on his snowboard .
option 3: a man in a blue shirt and green cap on a red snowboard .
option 4: a snowboarder flies through the air on a red snowboard in front of a snowy landscape .
option 5: a snowboarder landing on a pile of wood .


Output: a man in a white helmet is walking with a dog .
option 1: a black and brown dog is running between two cement <UNK> with snow .
option 2: a dog runs down the cold <UNK> .
option 3: black and tan small dog walking with <UNK> ears .
option 4: the black and brown dog walks toward the camera , in an enclosed , snowy area .
option 5: the brown and black dog is running through a snowy street .


Output: a man in a white helmet is walking with a dog .
option 1: two cyclists atop a hill as seen from below .
option 2: two cyclist wearing red shirts going over a red hill
option 3: two people in red outfits riding their bicycles down a dirt road .
option 4: two people ride their bikes down a dirt road .
option 5: two people riding their bikes on a hill .


Output: a man in a white shirt and jeans is standing in a field of tulips .
option 1: a guy is staring at a blonde lady in a blue jacket .
option 2: a man stands next to a lady with a dog hat .
option 3: a man walks next to the girl wearing headphones on a <UNK> day .
option 4: a man with glasses and a woman wearing a hat that looks like a dog are outside in the snow .
option 5: a woman with a <UNK> hat and headphones is in front of a man outside in the snow .


Output: a man in a white helmet is walking with a dog .
option 1: one football player wearing an orange jersey and one wearing a white jersey are set at the starting line .
option 2: two college football players about to face off .
option 3: two football players are getting ready to tackle each other
option 4: two football players , one in an orange jersey and one in a white jersey , play on the field .
option 5: two rival football players head to head .


Output: a man in a red jacket holds a small child up in the air .
option 1: a bicycle racer in full helmet speeds through a dim forest .
option 2: a biker dressed in white rides through the forest .
option 3: a mountain biker wearing white races through a turn .
option 4: a person on a motorbike , riding through the woods
option 5: riding a bike through the woods


Output: a man in a red jacket is watching something that is <UNK> off camera .
option 1: children playing in a fountain
option 2: four children are playing in a water fountain in a city plaza .
option 3: four children playing in water <UNK> .
option 4: little kids are jumping around in the sprinklers .
option 5: three children playing in a fountain .


Output: a man in a red jacket holds a small child up in the air .
option 1: a group of men play soccer .
option 2: a rugby player has <UNK> from the pack with the ball in his hands .
option 3: men playing a game in red uniforms
option 4: rugby player in red gets ready to toss the ball .
option 5: <UNK> are competing in a <UNK> game of rugby


Output: a man in a red jacket is watching something that is <UNK> off camera .
option 1: a boy in a green wetsuit takes his turn in a water sprinkler and a group of others wait .
option 2: a group of children play in a sprinkler outside .
option 3: children are taking turns running through a water sprinkler .
option 4: several kids run and play in the sprinklers .
option 5: the boy takes his turn through the sprinkler and jumps with joy .


Output: a man in a red jacket holds a small child up in the air .
option 1: a brown and white dog is running through woodland .
option 2: a large brown , white , and grey dog is running through a forest covered in leaves .
option 3: a tan long - haired dog is running in the forest
option 4: brown and white dog running through fall leaves on the ground .
option 5: the brown dog bounds through the leaves and the <UNK> .


Output: a man in a white helmet is walking with a dog .
option 1: a dog chewing a blue rope while two men watch from behind .
option 2: black dog with blue string in its mouth .
option 3: two men handle a grey dog with a blue <UNK> rope in its mouth .
option 4: two men lift a brown dog up so it can grab a large blue rope hanging down a black wall .
option 5: two men under a dog hanging by its mouth from a rope .


Output: a man in a red jacket holds a small child up in the air .
option 1: a hockey goalie in a red uniform takes a drink from a water bottle .
option 2: a hockey goalie taking a drink of <UNK> .
option 3: a hockey player in a red jersey taking a drink out of a sports bottle
option 4: an ice hockey player takes water from a green bottle .
option 5: hockey player in red uniform drinking from a green water bottle .


Output: a man in a red jacket holds a small child up in the air .
option 1: a person in a yellow helmet rock climbing while another person waits on the ground with the <UNK> .
option 2: a person rock climbing and another man on the ground holding the rope .
option 3: one person is rock climbing while another person is holding the rope and watching from below .
option 4: two people are rock climbing .
option 5: two rock climbers near the base of a rock face .


Output: a man in a white helmet is walking with a dog .
option 1: a woman in a green wig hula - hoops in front of a crowd .
option 2: a woman with green hair hula - hoops in a flowered orange top .
option 3: a woman with green hair hula hoops before a crowd .
option 4: a woman with green hair is hula hooping .
option 5: the girl with green hair and a green glove is swinging a hula hoop .


Output: a man in a red jacket holds a small child up in the air .
option 1: a cyclist wearing a red helmet is riding on the pavement .
option 2: a girl is riding a bike on the street while wearing a red helmet .
option 3: a person on a bike wearing a red helmet , riding down a street .
option 4: a woman wears a red helmet and blue shirt as she goes for a bike ride in the shade .
option 5: person in blue shirt and red helmet riding bike down the road


Output: a man in a white shirt and jeans stands next to a woman in a blue shirt .
option 1: two black and white dogs on the sandy beach fighting over a long stick
option 2: two dogs carry the same stick in their mouths as they walk on the sand .
option 3: two dogs holding one stick on a beach .
option 4: two dogs on the beach fight over a stick .
option 5: two dogs on wet sand , each biting the same stick .


{'1-gram': 0.4137175910397217, '2-gram': 0.19934879436869785, '3-gram': 0.0933133018758526, '4-gram': 0.05610834690917626}

Validition result

Output: a man in a red shirt is riding a bicycle on a track .
option 1: a man in a <UNK> <UNK> outfit is sitting on a <UNK> motorcycle .
option 2: a man in wearing a mask and black leather is sitting on a motorcycle on the street .
option 3: a person on a motorcycle wearing full leather with a <UNK> on it and <UNK> mask .
option 4: man in black leather on a motorcycle .
option 5: man in <UNK> - <UNK> leather outfit sitting <UNK> a red motorcycle .


Output: a boat sailing past the <UNK> sun .
option 1: a brown dog <UNK> up a <UNK> from a stone surface .
option 2: a brown puppy chewing on a stick .
option 3: a dog with big ears is holding a small stick in its mouth .
option 4: a puppy has a stick in its mouth .
option 5: a puppy with big ears chewing on a stick .


Output: a man drives a vehicle along a waterway lined with people .
option 1: a man <UNK> , <UNK> his off hand into the water .
option 2: a person leaning over while <UNK> .
option 3: a shirtless man is leaning back while <UNK> .
option 4: a <UNK> <UNK> the water with his <UNK> hand .
option 5: a young man <UNK> in the ocean .


Output: a boat sailing past the <UNK> sun .
option 1: a lone bicyclist jumping on his bike in front of a <UNK> wall .
option 2: a man doing tricks on a bike
option 3: a man <UNK> the <UNK> of his bicycle while he jumps .
option 4: a man on a bicycle in jeans is doing an air track along a dirt track with graffiti nearby .
option 5: a man performs bike tricks by a <UNK> .


Output: a boat sailing past the <UNK> sun .
option 1: a black dog running along the sandy <UNK> of a body of water while two ducks swim nearby .
option 2: a black dog running along the water
option 3: a dog and two birds .
option 4: a dog runs along the shore of a pond with two <UNK> <UNK> swimming .
option 5: dog running on shore beside two <UNK> in water


Output: a boat sailing past the <UNK> sun .
option 1: people kayaking in snowy and <UNK> <UNK> .
option 2: snowy mountains are in the background of three ocean kayakers .
option 3: three kayakers are traveling in the water alongside <UNK> mountains .
option 4: three men kayaking with mountains in the background
option 5: three people on red boats paddle through water with mountains in the background .


Output: a man and a woman are walking past benches that overlook the water .
option 1: a girl in pink pants running , an <UNK> <UNK> <UNK> in the background .
option 2: the girl dressed in purple walks through the <UNK> , <UNK> by a girl in pink .
option 3: two children <UNK> the <UNK> of a building .
option 4: two girls , one running away from the wall .
option 5: two little girls play around an old <UNK> building .


Output: a baby in a bouncy seat and a glove .
option 1: a girl playing is a pile of colorful <UNK> .
option 2: a little girl plays in a ball <UNK> .
option 3: a little girl plays in a <UNK> of colorful <UNK> .
option 4: a small girl is playing in a ball <UNK>
option 5: a young girl with a white shirt and pink shorts rolling around in some plastic <UNK> <UNK>


{'1-gram': 0.42821299112302436, '2-gram': 0.18037347573388168, '3-gram': 0.09790965196481695, '4-gram': 0.06826331698207638}


=> Saving checkpoint
epoch: [4/99]
Example 1 CORRECT: Dog on a beach by the ocean
Example 1 OUTPUT: <SOS> a man is sitting on a boat in the sunset . <EOS>
Example 2 CORRECT: Child holding red frisbee outdoors
Example 2 OUTPUT: <SOS> a young boy in a striped shirt and jeans runs in a field of grass . <EOS>
Example 3 CORRECT: Bus driving by parked cars
Example 3 OUTPUT: <SOS> a man in a yellow hi - viz jacket is drilling into a group of people . <EOS>
Example 4 CORRECT: A small boat in the ocean
Example 4 OUTPUT: <SOS> a boat sailing past the <UNK> sun . <EOS>
Example 5 CORRECT: A cowboy riding a horse in the desert
Example 5 OUTPUT: <SOS> a boat sailing past the <UNK> sun . <EOS>

Train result

Output: a man in a white helmet is standing by a <UNK> in a <UNK> .
option 1: a person in a red jacket snowboarding .
option 2: a person wearing a red coat snowboards down a snowy hill .
option 3: a snowboarder in red rides down the slope .
option 4: a snowboarder wearing a red jacket is boarding down the snowy hill .
option 5: the snowboarder is wearing a red jacket and black helmet .


Output: a man in a white helmet is standing by a <UNK> in a <UNK> .
option 1: a blue plane does <UNK> as it heads down to <UNK> .
option 2: a blue , red , and yellow airplane is flying through the air .
option 3: a blue , red , and yellow plane does a <UNK> in the air .
option 4: a colorful <UNK> plane leaves a <UNK> trail of smoke behind it .
option 5: a plane doing tricks in the air , leaving a smoke trail behind the plane .


Output: a man in a white helmet is standing by a <UNK> in a <UNK> .
option 1: a black dog and a black dog are fighting over a soccer ball .
option 2: two dogs are gnawing at a blue and yellow ball .
option 3: two dogs play and fight over a blue ball .
option 4: two dogs playing with a ball .
option 5: two dogs playing with a blue and green ball .


Output: a man in a white helmet is standing by a <UNK> in a <UNK> .
option 1: a hiker posing for a photo in an <UNK> mountain landscape .
option 2: a man in a green hat poses in the mountains
option 3: a man in a green hat standing on a slope in front of a mountain .
option 4: a man in camouflage hiking gear , wearing a green <UNK> is standing in the <UNK> .
option 5: a person in hiking gear is standing on a hill with a mountain in the background .


Output: a man in a white helmet is standing by a <UNK> in a <UNK> .
option 1: a goalie in a blue jersey catches a soccer ball by a player in a red jersey .
option 2: a man catches a ball in front of a large crowd .
option 3: a soccer goalie blocking a shot next to a player from the opposite team .
option 4: there is a goalie in the air making a <UNK> in a game .
option 5: two soccer players are playing soccer .


Output: a man in a white helmet is standing by a <UNK> in a <UNK> .
option 1: a brown and white dog is running through woodland .
option 2: a large brown , white , and grey dog is running through a forest covered in leaves .
option 3: a tan long - haired dog is running in the forest
option 4: brown and white dog running through fall leaves on the ground .
option 5: the brown dog bounds through the leaves and the <UNK> .


Output: a man in a white helmet is standing by a <UNK> in a <UNK> .
option 1: a group of 5 girls in school uniforms are standing together and waiting to cross the street while one girl takes a picture .
option 2: a group of people are standing together outside as one takes a photograph .
option 3: a group of school girls are standing together while one is taking pictures .
option 4: a group of young women stand together .
option 5: girls in school uniforms .


Output: a man in a white helmet is standing by a <UNK> in a <UNK> .
option 1: a child in a green cap sits at a boats <UNK> .
option 2: a little kid dressed in blue , playing in a boat ,
option 3: a small child in a cap is sitting on a large seat with hands through a steering wheel .
option 4: a small child in a <UNK> car .
option 5: the boy has his arms through the steering wheel


Output: a man in a white helmet is standing by a <UNK> in a <UNK> .
option 1: a man in a blue shirt passes under the <UNK> <UNK> .
option 2: a man in a blue shirt runs under a blue road marker in a race .
option 3: a runner in a blue shirt and blue shorts participates in a race .
option 4: a runner jogging on a city street past a <UNK> advertisement banner .
option 5: man running down a street in a marathon .


Output: a man in a white helmet is standing by a <UNK> in a <UNK> .
option 1: a boy in a tank top skateboarding .
option 2: a boy skateboards down the street .
option 3: a kid in a striped tank top skateboards down the street
option 4: a man wearing shorts and a tank top riding a skateboard .
option 5: a young man skateboarding on a street wearing a black and white striped shirt


Output: a man in a white helmet is standing by a <UNK> in a <UNK> .
option 1: a girl in a yellow dress is standing on the pavement outside a store .
option 2: a girl in a yellow dress looks at a restaurant building .
option 3: a girl in yellow dress near a restaurant .
option 4: a little girl in a yellow dress is standing in front of a restaurant .
option 5: a young girl wearing a yellow dress , gray pants and flip flops is standing in front of a restaurant .


Output: a man in a white helmet is standing by a <UNK> in a <UNK> .
option 1: a man sits at a table in front of a computer wearing headphones .
option 2: a man with headphones on is looking at a computer screen while sitting at a table .
option 3: a young man is sitting at a table with a laptop and earphones .
option 4: a young man with headphones looking at a <UNK> laptop computer .
option 5: man sitting with headphones at a laptop .


Output: a man in a white helmet is standing by a <UNK> in a <UNK> .
option 1: a boy in a striped <UNK> shirt points .
option 2: a boy points at something behind the camera .
option 3: a little boy wearing a striped shirt is pointing at something .
option 4: a young boy in a striped shirt pointing beyond the camera .
option 5: a young boy wearing a striped shirt points into the distance .


Output: a man in a white helmet is standing by a <UNK> in a <UNK> .
option 1: a boy stands on a rocky mountain .
option 2: a child on a rock formation .
option 3: a child sitting on a rock formation .
option 4: a little boy <UNK> on top of a big rock .
option 5: a person in a red jacket on top of rocks .


Output: a man in a white helmet is standing by a <UNK> in a <UNK> .
option 1: a bearded man holding a sign .
option 2: a homeless man carries a sign that says " <UNK> " .
option 3: a homeless person asking for help
option 4: a man in a black hoodie holding a small sign .
option 5: a man in a black hoodie is holding a paper sign .


Output: a man in a white helmet is standing by a <UNK> in a <UNK> .
option 1: a man <UNK> control of his watercraft .
option 2: a man <UNK> on the water .
option 3: a windsurfer falls off his boat in the ocean .
option 4: the man was sailing in the ocean and <UNK> off his boat .
option 5: the <UNK> is falling in the water .


{'1-gram': 0.5009420159182475, '2-gram': 0.27330640934773326, '3-gram': 0.14760484412186414, '4-gram': 0.09203573932819516}


!wget https://bin.equinox.io/c/4VmDzA7iaHb/ngrok-stable-linux-amd64.zip
!unzip ngrok-stable-linux-amd64.zip

import os
LOG_DIR = "runs"
os.makedirs(LOG_DIR, exist_ok=True)
get_ipython().system_raw(
    'tensorboard --logdir {} --host 0.0.0.0 --port 6006 &'
    .format(LOG_DIR)
)
get_ipython().system_raw('./ngrok http 6006 &')
!curl -s http://localhost:4040/api/tunnels | python3 -c \
    "import sys, json; print(json.load(sys.stdin)['tunnels'][0]['public_url'])"

--2020-11-26 13:53:26--  https://bin.equinox.io/c/4VmDzA7iaHb/ngrok-stable-linux-amd64.zip
Resolving bin.equinox.io (bin.equinox.io)... 52.22.13.178, 52.21.175.83, 34.195.187.253, ...
Connecting to bin.equinox.io (bin.equinox.io)|52.22.13.178|:443... connected.
HTTP request sent, awaiting response... 200 OK
Length: 13773305 (13M) [application/octet-stream]
Saving to: ‘ngrok-stable-linux-amd64.zip.13’

ngrok-stable-linux- 100%[===================>]  13.13M  33.7MB/s    in 0.4s    

2020-11-26 13:53:26 (33.7 MB/s) - ‘ngrok-stable-linux-amd64.zip.13’ saved [13773305/13773305]

Archive:  ngrok-stable-linux-amd64.zip
replace ngrok? [y]es, [n]o, [A]ll, [N]one, [r]ename: y
  inflating: ngrok                   
https://62c81ac1d99e.ngrok.io


# #__________________RUN TO ACTIVATE TENSORBOARD____________________
%reload_ext tensorboard
%tensorboard --logdir {LOG_DIR}


save_point = torch.load("/content/drive/MyDrive/Colab Notebooks/imageCaptions/checkpoint_1120.pth.tar")
embed_size, hidden_size, num_layers = 512, 512, 2
vocab_size = len(train_dataset.vocab)
model =  EncoderDecoder(embed_size, hidden_size, vocab_size, num_layers).to(device) 
model.load_state_dict(save_point["state_dict"]);

Downloading: "https://download.pytorch.org/models/resnet152-b121ed2d.pth" to /root/.cache/torch/hub/checkpoints/resnet152-b121ed2d.pth


test_loader, test_dataset = my_get_loader(
                                    root_folder=ROOT_FOLDER,
                                    df= pd.DataFrame.from_dict(split['test']),
                                    transform=transform,
                                    batch_size=16,
                                    num_workers=4,
                                    shuffle=False)


test_prediction = {
            'image id':[],
            'kpi': [],
            'captions list': [],
            'output': []
    }

model.eval()
with torch.no_grad():
    for idx, data in tqdm(enumerate(test_loader), total=len(test_loader)):

        _, batch_result = blue_score(data, test_dataset) #calculate BLUE score of the batch [16X]   
        for key, values in batch_result.items():
            for l in values:
                test_prediction[f'{key}'].append(l)

/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:10: TqdmDeprecationWarning: This function will be removed in tqdm==5.0.0
Please use `tqdm.notebook.tqdm` instead of `tqdm.tqdm_notebook`
  # Remove the CWD from sys.path while we load stuff.


result_df = pd.DataFrame.from_dict(test_prediction)


n_grams = {
    '1-gram':[],
    '2-gram':[],
    '3-gram':[],
    '4-gram':[]
}
for res in test_prediction['kpi']:
    for key, val in res.items():
        n_grams[f'{key}'].append(val)
n_grams = pd.DataFrame.from_dict(n_grams)
n_grams


keys = n_grams.keys()
plt.figure(figsize=[15, 9], dpi=100)
labels = ['1 gram', '2 gram', '3 gram', '4 gram']
plt.boxplot([n_grams[key] for key in keys], labels=labels)#, fontsize=20)
plt.xticks()
plt.show()


def show_test_image(image_id):
    plt.figure(figsize=[15, 9])
    img = plt.imread(os.path.join(ROOT_FOLDER, image_id))
    img = cv2.resize(img, (600, 500))

    #show output
    output =str(" ".join(*result_df.loc[result_df['image id'] == image_id]['output'].to_list()))
    plt.text(2, 10, output, fontweight='bold', fontsize=12, bbox=dict(facecolor='red', alpha=0.5))

    #show captions
    eps = 20 
    for caption in zip(*result_df.loc[result_df['image id'] == image_id]['captions list']):

        plt.text(2, 10 + eps, " ".join(caption[0]), fontweight='bold', fontsize=12, bbox=dict(facecolor='green', alpha=0.5))
        eps += 20
    
    #show BLUE score
    eps = 0
    values = result_df.loc[result_df['image id'] == image_id]['kpi'].values[0]
    for gram, val in values.items():
        n = gram.replace('-', ' ')
        plt.text(480, 420 + eps, f'{n}: {round(val, 5)}', fontweight='bold', fontsize=12, bbox=dict(facecolor='green', alpha=0.5))
        eps += 20
    plt.imshow(img)
    plt.axis('off')
    plt.show()


n_grams.describe()


#LRB - Low Range Boundry
#URB - Upper Range Boundry

mean, std = n_grams.describe()['1-gram'][['mean', 'std']][:2]
LRB = mean - std
URB = mean + std

low_indexes = n_grams.loc[n_grams['1-gram'] < LRB].index
meduim_indexes = n_grams.loc[(n_grams['1-gram'] >= LRB) & (n_grams['1-gram'] < URB)].index
hight_indexes = n_grams.loc[n_grams['1-gram'] >= URB].index


print(f'The mean value of 1 gram BLUE score is {round(mean, 5)} with standard deviation of {round(std, 5)}.\
\nMost of the values obtained will be between {round(LRB, 5)} and {round(URB, 5)}, the rest will called outliers')
print(f'Distribution result:\noutliers {len(low_indexes) + len(hight_indexes)} with , in range {len(meduim_indexes)}')
print(f'There is {len(low_indexes)} images lowwer then the lowwer standard deviation bound and {len(hight_indexes)} images greater than the upper bound std')

The mean value of 1 gram BLUE score is 0.48832 with standard deviation of 0.15025.
Most of the values obtained will be between 0.33807 and 0.63857, the rest will called outliers
Distribution result:
outliers 294 with , in range 706
There is 155 images lowwer then the lowwer standard deviation bound and 139 images greater than the upper bound std


from random import sample, seed


seed(0)
for i in sample(low_indexes.tolist(), 4):
    show_test_image(result_df.iloc[i]['image id'])


seed(5)
for i in sample(meduim_indexes.tolist(), 4):
    show_test_image(result_df.iloc[i]['image id'])


seed(3) 
for i in sample(hight_indexes.tolist(), 4):
    show_test_image(result_df.iloc[i]['image id'])


show_test_image('241347760_d44c8d3a01.jpg')
show_test_image('2654514044_a70a6e2c21.jpg')
show_test_image('3072172967_630e9c69d0.jpg')
show_test_image('2847615962_c330bded6e.jpg')

	sentids	imgid	sentences	split	filename
0	[0, 1, 2, 3, 4]	0	[{'tokens': ['a', 'black', 'dog', 'is', 'runni...	train	2513260012_03d33305cf.jpg
1	[5, 6, 7, 8, 9]	1	[{'tokens': ['a', 'little', 'baby', 'plays', '...	train	2903617548_d3e38d7f88.jpg
2	[10, 11, 12, 13, 14]	2	[{'tokens': ['a', 'brown', 'dog', 'in', 'the',...	train	3338291921_fe7ae0c8f8.jpg
3	[15, 16, 17, 18, 19]	3	[{'tokens': ['a', 'brown', 'dog', 'is', 'runni...	train	488416045_1c6d903fe0.jpg
4	[20, 21, 22, 23, 24]	4	[{'tokens': ['a', 'black', 'and', 'white', 'do...	train	2644326817_8f45080b87.jpg
5	[25, 26, 27, 28, 29]	5	[{'tokens': ['a', 'cyclist', 'wearing', 'a', '...	train	218342358_1755a9cce1.jpg
6	[30, 31, 32, 33, 34]	6	[{'tokens': ['a', 'man', 'dressed', 'in', 'a',...	train	2501968935_02f2cd8079.jpg
7	[35, 36, 37, 38, 39]	7	[{'tokens': ['a', 'boy', 'wearing', 'a', 'red'...	train	2699342860_5288e203ea.jpg
8	[40, 41, 42, 43, 44]	8	[{'tokens': ['a', 'girl', 'in', 'a', 'white', ...	train	2638369467_8fc251595b.jpg
9	[45, 46, 47, 48, 49]	9	[{'tokens': ['a', 'skier', 'in', 'a', 'yellow'...	train	2926786902_815a99a154.jpg

	sentids	imgid	sentences	split	filename
0	[30000, 30001, 30002, 30003, 30004]	6000	[{'tokens': ['the', 'boy', 'laying', 'face', '...	val	2090545563_a4e66ec76b.jpg
1	[30005, 30006, 30007, 30008, 30009]	6001	[{'tokens': ['a', 'boy', 'in', 'a', 'blue', 't...	val	3393035454_2d2370ffd4.jpg
2	[30010, 30011, 30012, 30013, 30014]	6002	[{'tokens': ['a', 'lady', 'walking', 'her', 'd...	val	3695064885_a6922f06b2.jpg
3	[30015, 30016, 30017, 30018, 30019]	6003	[{'tokens': ['a', 'big', 'black', 'dog', 'jump...	val	1679557684_50a206e4a9.jpg
4	[30020, 30021, 30022, 30023, 30024]	6004	[{'tokens': ['two', 'woman', 'climbing', 'rock...	val	3582685410_05315a15b8.jpg
5	[30025, 30026, 30027, 30028, 30029]	6005	[{'tokens': ['a', 'child', 'is', 'laying', 'in...	val	1579798212_d30844b4c5.jpg
6	[30030, 30031, 30032, 30033, 30034]	6006	[{'tokens': ['a', 'group', 'of', 'people', 'ar...	val	269650644_059a84ece5.jpg
7	[30035, 30036, 30037, 30038, 30039]	6007	[{'tokens': ['four', 'children', 'are', 'playi...	val	2596619849_7b635dd310.jpg
8	[30040, 30041, 30042, 30043, 30044]	6008	[{'tokens': ['a', 'little', 'baby', 'skateboar...	val	2635164923_2a774f7854.jpg
9	[30045, 30046, 30047, 30048, 30049]	6009	[{'tokens': ['an', 'elderly', 'middle', 'easte...	val	3583321426_f373c52161.jpg

	sentids	imgid	sentences	split	filename
0	[35000, 35001, 35002, 35003, 35004]	7000	[{'tokens': ['the', 'dogs', 'are', 'in', 'the'...	test	3385593926_d3e9c21170.jpg
1	[35005, 35006, 35007, 35008, 35009]	7001	[{'tokens': ['a', 'brown', 'and', 'white', 'do...	test	2677656448_6b7e7702af.jpg
2	[35010, 35011, 35012, 35013, 35014]	7002	[{'tokens': ['a', 'man', 'and', 'a', 'woman', ...	test	311146855_0b65fdb169.jpg
3	[35015, 35016, 35017, 35018, 35019]	7003	[{'tokens': ['a', 'couple', 'of', 'people', 's...	test	1258913059_07c613f7ff.jpg
4	[35020, 35021, 35022, 35023, 35024]	7004	[{'tokens': ['a', 'man', 'is', 'wearing', 'a',...	test	241347760_d44c8d3a01.jpg
5	[35025, 35026, 35027, 35028, 35029]	7005	[{'tokens': ['a', 'brown', 'dog', 'running'], ...	test	2654514044_a70a6e2c21.jpg
6	[35030, 35031, 35032, 35033, 35034]	7006	[{'tokens': ['a', 'girl', 'with', 'dark', 'bro...	test	2339106348_2df90aa6a9.jpg
7	[35035, 35036, 35037, 35038, 35039]	7007	[{'tokens': ['a', 'dog', 'with', 'its', 'mouth...	test	256085101_2c2617c5d0.jpg
8	[35040, 35041, 35042, 35043, 35044]	7008	[{'tokens': ['a', 'black', 'dog', 'emerges', '...	test	280706862_14c30d734a.jpg
9	[35045, 35046, 35047, 35048, 35049]	7009	[{'tokens': ['a', 'player', 'from', 'the', 'wh...	test	3072172967_630e9c69d0.jpg

	1-gram	2-gram	3-gram	4-gram
0	0.307692	0.050637	0.029600	0.021973
1	0.714286	0.468807	0.335789	0.240221
2	0.384615	0.179029	0.068119	0.041316
3	0.411765	0.050730	0.026753	0.018710
4	0.785714	0.549725	0.138793	0.069172
...	...	...	...	...
995	0.545455	0.073855	0.040572	0.029502
996	0.400000	0.066667	0.039424	0.029847
997	0.900000	0.707107	0.400535	0.172860
998	0.573112	0.228333	0.083005	0.049981
999	0.363636	0.060302	0.035491	0.026658

	1-gram	2-gram	3-gram	4-gram
count	1000.000000	1000.000000	1000.000000	1000.000000
mean	0.488320	0.239379	0.129634	0.078967
std	0.150254	0.166527	0.130623	0.094952
min	0.125000	0.029525	0.017825	0.012918
25%	0.383352	0.067420	0.040332	0.029847
50%	0.477688	0.217930	0.079548	0.046118
75%	0.571429	0.333372	0.169062	0.082487
max	1.000000	1.000000	1.000000	1.000000

Data Collection¶

Understanding the data¶

Show random example¶

image with her 5 captions¶

Transforms the images to shape 224X224 and normalize to ImageNet values¶

Network Architecture¶

Model parameters¶

Train¶

Inference section¶

Test result¶

Unsuccessful prediction¶

Partly success¶

Accurate description of the image¶